<?php
	require_once "bin/functions.php";
	require_once "Database.php";
	require_once "wordpress-config.php";
	require_once "bin/logger.php";
	require_once "bin/mink/vendor/autoload.php";
	require_once 'DataScraperToWordpress.php';
	
	//EXECUTION TIME
	//ini_set('max_execution_time', 300);
	
	
	class DataScraper
	{
		//execute the dataminer on each url
		public static function executeDataScraping($links, $lookForNewLinks = false)
		{
			global $mink_session;
			$mink_session = startMink();
			$array_data_retrieved = array();
			for($i = 0; $i < count($links); $i++)
			{
				addToArray($array_data_retrieved, !$lookForNewLinks ? self::DataScrape($links, $i) : DataMine($links, $i, null, $lookForNewLinks));
			}
			$mink_session->stop();
			return $array_data_retrieved;
		}
		
		private static function DataScrape(&$links, $url_number = 0, $php_scripts = null, $lookForNewLinks = false)
		{
			if(!isset($lookForNewLinks))
				$lookForNewLinks = false;
			$linkscount = count($links);
			$main_program = TRUE;
			$url = NULL;
			
			//initialize url
			if(gettype($links) == "array")
			{
				$url = $links[$url_number];
			}
			elseif(gettype($links) == "string")
			{
				$url = $links;
			}
			else
			{
				return null;
			}
			
			//for web-link developer
			if($php_scripts != null)
			{
				$main_program = FALSE;
			}
			
			if(is_valid_url($url))
			{
				$url_pre = NULL;
				$url_post = NULL;
				$url_type = NULL;
		
				//preg_match("/(http|https)\:\/\/([".LinkRegex()."]*)/", $url, $url_pre);
				//preg_match("/([a-zA-Z0-9_\.\-]*)([".LinkRegex()."]*)/"
				
				$url_split = explode("/", $url);
				if($url_split[0] == "http:" || $url_split[0] == "https:")
				{
					$url_pre = $url_split[2];
					for($i = 3; $i < count($url_split); $i++)
					{
						$url_post = $url_post . "/" . $url_split[$i];
					}
				}
				else
				{
					$url = "https://" . $url;
					$url_pre = $url_split[0];
					for($i = 1; $i < count($url_split); $i++)
					{
						$url_post = $url_post . "/" . $url_split[$i];
					}
				}
				/*** gather required data from website ***/
				//make call to website
				$url_value = file_get_contents($url);
				$url_header = getHeader($url);//$http_response_header;
				$url_type = gettype($url_header["Content-Type"]) == "string" ? $url_header["Content-Type"] : $url_header["Content-Type"][0];
				preg_match("/(xml|json|html)/", $url_type, $matches);
				$url_type = $matches[1];
		
				//execute mining process using php scripts from database
				$requiredText = "+_!@#&*#requiredText";
		
				global $dataset_url;
				global $extraMetadataDetails;
				global $size;
				global $records;
				global $protocol;
				global $organizationName;
				global $organizationWebsite;
				global $organizationDescription;
				global $organizationContactPerson;
				global $organizationEmail;
				global $organizationPhoneNumber;
				global $dataset_update_time;
				global $dataset_type;
				global $dataset_demo_url;
				global $dataset_screenshot;
				global $blogTitle;
				global $blogContent;
				global $blogExcerpt;
				global $dataset_categories;
				global $dataset_tags;
				global $file_types;
		
				$dataset_url = $requiredText;
				$extraMetadataDetails = "";
				$size = "1";
				$records = "1";
				$protocol = "";
				$organizationName = $requiredText;
				$organizationWebsite = "";
				$organizationDescription = "";
				$organizationContactPerson = "";
				$organizationEmail = $requiredText;
				$organizationPhoneNumber = "";
				$dataset_update_time = 1;
				$dataset_type = 0;
				$dataset_demo_url = "";
				$dataset_screenshot = array();
				$blogTitle = $requiredText;
				$blogContent = "";
				$blogExcerpt = "";
				$dataset_categories = array();
				$dataset_tags = array();
				$file_types = array();
				//$postPassword???????;
				
				$data = array(
					"dataset_url" => $dataset_url,
					"extraMetadataDetails" => $extraMetadataDetails,
					"size" => $size,
					"records" => $records,
					"protocol" => $protocol,
					"organizationName" => $organizationName,
					"organizationWebsite" => $organizationWebsite,
					"organizationDescription" => $organizationDescription,
					"organizationContactPerson" => $organizationContactPerson,
					"organizationEmail" => $organizationEmail,
					"organizationPhoneNumber" => $organizationPhoneNumber,
					"dataset_update_time" => $dataset_update_time,
					"dataset_type" => $dataset_type,
					"dataset_demo_url" => $dataset_demo_url,
					"dataset_screenshot" => $dataset_screenshot,
					"blogTitle" => $blogTitle,
					"blogContent" => $blogContent,
					"blogExcerpt" => $blogExcerpt,
					"dataset_categories" => $dataset_categories,
					"dataset_tags" => $dataset_tags,
					"file_types" => $file_types
				);
		
				
				$data_array = array();
				$auto_build_data = true;
				
				if($main_program)
				{
					/*$isRealScript = null;
					foreach($php_scripts as $scripts)
					{
						eval(preparePHPScript($scripts));
						if($isRealScript != null)
							break;
					}*/
					require("scraping_scripts.php");
				}
				else
				{
					eval($php_scripts);
				}
				
				
				if($auto_build_data)
					$data = array(
						"dataset_url" => $dataset_url,
						"extraMetadataDetails" => $extraMetadataDetails,
						"size" => $size,
						"records" => $records,
						"protocol" => $protocol,
						"organizationName" => $organizationName,
						"organizationWebsite" => $organizationWebsite,
						"organizationDescription" => $organizationDescription,
						"organizationContactPerson" => $organizationContactPerson,
						"organizationEmail" => $organizationEmail,
						"organizationPhoneNumber" => $organizationPhoneNumber,
						"dataset_update_time" => $dataset_update_time,
						"dataset_type" => $dataset_type,
						"dataset_demo_url" => $dataset_demo_url,
						"dataset_screenshot" => $dataset_screenshot,
						"blogTitle" => $blogTitle,
						"blogContent" => $blogContent,
						"blogExcerpt" => $blogExcerpt,
						"dataset_categories" => $dataset_categories,
						"dataset_tags" => $dataset_tags,
						"file_types" => $file_types
					);
				$url_info = null;
				if(count($data_array) != 0)
				{
					$valid_array = true;
					foreach($data_array as $row)
					{
						if(!self::validateDataGathered($row))
						{
							if($main_program)
								return null;
							$valid_array = false;
						}
					}
					$url_info = NULL;
					if($valid_array)
					{
						$url_info = "Data Collected";
					}
					else
					{
						if($main_program)
							$data_array = array();
						$url_info = "No Data Collected";
					}
				}
				elseif(gettype($data["dataset_url"]) == "string")
				{
					if($main_program)
					{
						$url_info = NULL;
						if(self::validateDataGathered($data))
						{
							$url_info = "Data Collected";
						}
						else
						{
							//if(!$main_program)
							//	$data = NULL;
							//$url_info = "No Data Collected";
							return null;
						}
						array_push($data_array, $data);
					}
					else
					{
						$url_info = "Data Collected";
						array_push($data_array, $data);
					}
				}
				else
				{
					$data = null;
					$url_info = null;
				}
				$return_value = array();
				if(count($data_array)== 1)
				{
					array_push($return_value, array($data_array[0], $url_info));//, $url,  (($linkscount == count($links)) ? NULL : (((count($links)  - $linkscount) != 1 || $data_array[0][0][0]["dataset_url"] == "+_!@#&*#requiredText") ? NULL : $links[count($links)-1]))));
				}
				else
				{
					foreach($data_array as $row)
					{
						array_push($return_value, array($row, $url_info));//, $row == NULL ? $url : $row["dataset_url"],  (($linkscount == count($links)) ? NULL : (((count($links)  - $linkscount) != 1 || $data_array[0][0][0]["dataset_url"] == "+_!@#&*#requiredText") ? NULL : $links[count($links)-1]))));
					}
				}
				return $return_value;
			}
		}
		
		public static function validateDataGathered($data)
		{
			if(array_key_exists("dataset_url", $data))
			{
				return self::checkDataGathered($data);
			}
			else
			{
				$returnvalue = true;
				foreach($data as $row)
				{
					if(!self::checkDataGathered($row))
					{
						$returnvalue = false;
					}
				}
				return $returnvalue;
			}
		}
		
		private static function checkDataGathered($data)
		{
			if($data["dataset_url"] == "+_!@#&*#requiredText" || gettype($data["dataset_url"]) != "string" || $data["dataset_url"] == "")
				return false;
			if(gettype($data["extraMetadataDetails"]) != "string")
				return false;
			if(!(gettype($data["size"]) == "integer" || gettype($data["size"]) == "double" || gettype($data["size"]) == "string"))
				return false;
			if(!(gettype($data["records"]) == "string" || gettype($data["records"]) == "integer"))
				return false;
			if(gettype($data["protocol"]) != "string")
				return false;
			if($data["organizationName"] == "+_!@#&*#requiredText" || gettype($data["organizationName"]) != "string" || $data["organizationName"] == "")
				return false;
			if(gettype($data["organizationWebsite"]) != "string")
				return false;
			if(gettype($data["organizationDescription"]) != "string")
				return false;
			if(gettype($data["organizationContactPerson"]) != "string")
				return false;
			if($data["organizationEmail"] == "+_!@#&*#requiredText" || gettype($data["organizationEmail"]) != "string" || $data["organizationEmail"] == "")
				return false;
			if(gettype($data["organizationPhoneNumber"]) != "string")
				return false;
			if(gettype($data["dataset_update_time"]) != "integer")
				return false;
			if(gettype($data["dataset_type"]) != "integer")
				return false;
			if(gettype($data["dataset_demo_url"]) != "string")
				return false;
			if(gettype($data["dataset_screenshot"]) != "array")
				return false;
			if($data["blogTitle"] == "+_!@#&*#requiredText" || gettype($data["blogTitle"]) != "string" || $data["blogTitle"] == "")
				return false;
			if(gettype($data["blogContent"]) != "string")
				return false;
			if(gettype($data["blogExcerpt"]) != "string")
				return false;
			if(gettype($data["dataset_categories"]) != "array")
				return false;
			foreach($data["dataset_categories"] as $row)
				if(gettype($row) != "string")
					return false;
			if(gettype($data["dataset_tags"]) != "array")
				return false;
			foreach($data["dataset_tags"] as $row)
				if(gettype($row) != "string")
					return false;
			if(gettype($data["file_types"]) != "array")
				return false;
			foreach($data["file_types"] as $row)
				if(gettype($row) != "string")
					return false;
			return true;
		}
		
		public static function searchForLinks($search_array)
		{
			global $RealDataSetLinksStart;
			global $RealDataSetLinksEnd;
			
			$url_array = array();
			if($search_array)
			{
				foreach($search_array as $row)
				{
					if(array_key_exists("url", $row))
					{
			$each_row_url = $row["url"];
						array_push($url_array, $row["url"]);
					}
					elseif(array_key_exists("dataset_url", $row))
					{
			$each_row_url = $row["dataset_url"];
						array_push($url_array, $each_row_url);
					}
					elseif(array_key_exists(0, $row))
					{
						if(array_key_exists("dataset_url", $row[0]))
						{
			$each_row_url = $row[0]["dataset_url"];
							array_push($url_array, $each_row_url);
						}
					}
					/*$search_string = arrayToString($row);
					$array = explode($RealDataSetLinksStart, $search_string);
					for($i = 1; $i < count($array); $i++)
					{
						$urls = explode($RealDataSetLinksEnd,$array[$i])[0];
			$exploded_url_array = explode("REAL Data Set Links:",$urls);
			$exploded_url_array = $exploded_url_array[1];
						$urls = trim_array(explode(",",preg_replace(array("/\r/", "/\n/", "/\<br\>/"),array("",",", ","), $exploded_url_array)), true);
						addToArray($url_array, $urls);
					}*/
				}
			}
			return $url_array;
		}
		
		public static function emptyData()
		{
			global $dataset_url;
			global $extraMetadataDetails;
			global $size;
			global $records;
			global $protocol;
			global $organizationName;
			global $organizationWebsite;
			global $organizationDescription;
			global $organizationContactPerson;
			global $organizationEmail;
			global $organizationPhoneNumber;
			global $dataset_update_time;
			global $dataset_type;
			global $dataset_demo_url;
			global $dataset_screenshot;
			global $blogTitle;
			global $blogContent;
			global $blogExcerpt;
			global $dataset_categories;
			global $dataset_tags;
			global $file_types;
			
			$requiredText = "+_!@#&*#requiredText";
	
			$dataset_url = $requiredText;
			$extraMetadataDetails = "";
			$size = "1";
			$records = "1";
			$protocol = "";
			$organizationName = $requiredText;
			$organizationWebsite = "";
			$organizationDescription = "";
			$organizationContactPerson = "";
			$organizationEmail = $requiredText;
			$organizationPhoneNumber = "";
			$dataset_update_time = 1;
			$dataset_type = 0;
			$dataset_demo_url = "";
			$dataset_screenshot = array();
			$blogTitle = $requiredText;
			$blogContent = "";
			$blogExcerpt = "";
			$dataset_categories = array();
			$dataset_tags = array();
			$file_types = array();
	
			
		}
		
		public static function createData()
		{
			global $dataset_url;
			global $extraMetadataDetails;
			global $size;
			global $records;
			global $protocol;
			global $organizationName;
			global $organizationWebsite;
			global $organizationDescription;
			global $organizationContactPerson;
			global $organizationEmail;
			global $organizationPhoneNumber;
			global $dataset_update_time;
			global $dataset_type;
			global $dataset_demo_url;
			global $dataset_screenshot;
			global $blogTitle;
			global $blogContent;
			global $blogExcerpt;
			global $dataset_categories;
			global $dataset_tags;
			global $file_types;
			
			return array(
				"dataset_url" => $dataset_url,
				"extraMetadataDetails" => $extraMetadataDetails,
				"size" => $size,
				"records" => $records,
				"protocol" => $protocol,
				"organizationName" => $organizationName,
				"organizationWebsite" => $organizationWebsite,
				"organizationDescription" => $organizationDescription,
				"organizationContactPerson" => $organizationContactPerson,
				"organizationEmail" => $organizationEmail,
				"organizationPhoneNumber" => $organizationPhoneNumber,
				"dataset_update_time" => $dataset_update_time,
				"dataset_type" => $dataset_type,
				"dataset_demo_url" => $dataset_demo_url,
				"dataset_screenshot" => $dataset_screenshot,
				"blogTitle" => $blogTitle,
				"blogContent" => $blogContent,
				"blogExcerpt" => $blogExcerpt,
				"dataset_categories" => $dataset_categories,
				"dataset_tags" => $dataset_tags,
				"file_types" => $file_types
			);
		}
		
		public static function StringToBytes($string)
		{
			$string = trim($string);
			preg_match("/[0-9]*/",$string, $number);
			if(count($number[0]) == 0)
				return "1";
			preg_match("/[a-z]/",strtolower($string), $size);
			if(count($size) != 0)
			{
				switch ($size[0]) {
				    case "b":
				        return (int)$number[0] * 1;
				        break;
				    case "k":
				        return (int)$number[0] * 1000;
				        break;
				    case "m":
				        return (int)$number[0] * 1000000;
				        break;
				    case "g":
				    	return (int)$number[0] * 1000000000;
				    	break;
				}
			}
		}
	}
?>